package com.play.crawler;

import java.util.regex.Pattern;

import com.play.parse.MyParser;
import com.play.parse.impl.M18Handler;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.Parser;
import edu.uci.ics.crawler4j.url.WebURL;

public class M18Crawler extends WebCrawler {



	private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g" + "|png|tiff?|mid|mp2|mp3|mp4"
			+ "|wav|avi|mov|mpeg|ram|m4v|pdf" + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

	/**
	 * You should implement this function to specify whether the given url
	 * should be crawled or not (based on your crawling logic).
	 */
	@Override
	public boolean shouldVisit(WebURL url) {
		String href = url.getURL().toLowerCase();
		//return !FILTERS.matcher(href).matches() && href.startsWith("http://www.ics.uci.edu/");
		//boolean domin = href.startsWith("http://www.aimer.com.cn/");
		
		return !FILTERS.matcher(href).matches() ;
	}

	/**
	 * This function is called when a page is fetched and ready to be processed
	 * by your program.
	 */
	@Override
	public void visit(Page page) {
	}

	@Override
	protected Parser findMyParser() {
		return new MyParser(myController.getConfig(),new M18Handler());
	}
}
